import json
import random
import time

import pymysql
from bs4 import BeautifulSoup
from facade.loggerfacade import get_streamlogger
from facade.mysqlfacade import MysqlUtiles
from xjlibrary.mprocesspoll.MThreadingRun import MThreadingRun
from xjlibrary.mrequest.baserequest import MProxyRequest, USER_AGENTS
from xjlibrary.myredis.myredisclient import getDataFromRedis
from xjlibrary.our_file_dir import BaseDir
from xjlibrary.tools.BaseString import BaseString

curPath = BaseDir.get_file_dir_absolute(__file__)
configfile = BaseDir.get_new_path(curPath, "db.ini")

BookBaseUrl = "https://pubs.acs.org"

HEADERS = {'User-Agent': random.choice(USER_AGENTS)}

keytransformdicts = {"port": "db_port", "host": "db_host", "passwd": "db_pw", "user": "db_user",
                     "db": "db_name",
                     "chartset": "db_charset"}

logger = get_streamlogger()
myutil = MysqlUtiles(configfile, "db", keytransformdicts=keytransformdicts, logger=logger)


def SelectListFromDB():
    sql = "select `title`,`url`,`doi`,`jsonmsg` from `ebook` where stat=0"
    rows = myutil.SelectFromDB(sql)
    return rows


def getBookInfo(r):
    html = r.text
    bs = BeautifulSoup(html, "lxml")
    div_tag = bs.find("div", class_="meta1")
    div_chrild_tas = div_tag.find_all("div")[0]
    sub_tag = bs.find("div", class_="subtitles")
    if sub_tag:
        substrings = sub_tag.get_text().strip()
    else:
        substrings = ""
    # # 作者dicts
    # dicts = {}
    # 机构dicts
    dicts1 = {}
    authors = ""
    author = ""
    key = ""
    if div_chrild_tas.find("Editor") != -1:
        for tag in div_chrild_tas.contents:
            if tag.name == "span":
                author = tag.get_text().strip()
                print("author is :" + author)
            if tag.name == "sup":
                if key == "":
                    key = tag.get_text().strip()
                else:
                    key = key + "," + tag.get_text().strip()
                print("key is :" + key)
            if tag == ", ":
                if key or author:
                    if key != "":
                        authors += author + "[" + key + "]" + ";"
                    else:
                        authors += author + ";"
                author = ""
                key = ""
        if key or author:
            if key != "":
                authors += author + "[" + key + "]" + ";"
            else:
                authors += author + ";"
        authors = BaseString.r_strip_one_char(authors)

        div_tag = bs.find("div", class_="affiliations")
        if div_tag:
            for tag in div_tag.contents:
                num = tag.sup.get_text()
                strings = tag.contents[1].strip()
                dicts1[num] = strings
    # 版权
    copyright = bs.find("span", class_="NLM_publisher-name").get_text().strip()
    # 赞助
    spon = bs.find("div", class_="sponsors")
    if spon:
        spon = spon.get_text().strip()
    else:
        spon = ""
    spon = spon.replace("Sponsoring Divisions", "").replace(":", "").strip()

    author_institution = ""
    for key, value in dicts1.items():
        author_institution += "[" + key + "]" + value + ";"
    author_institution = BaseString.r_strip_one_char(author_institution)

    return copyright, spon, authors, author_institution, substrings


def down_page(result_queue, row, list_proxy):
    # if row[1] != '/isbn/9780841200029':
    #     return
    while True:
        title = row[0]
        url1 = row[1]
        doi = row[2]
        jsonmsg = row[3]
        url = BookBaseUrl + url1
        BoolResult, errString, r = MProxyRequest(url,
                                                 Feature="bookBox",
                                                 HEADERS=HEADERS,
                                                 proxyRingList=list_proxy,
                                                 verify=False,
                                                 timeout=(30, 60))
        if not BoolResult:
            logger.error(errString)
            continue
        bookInfo = getBookInfo(r)
        copyright, spon, authors, author_institution, substrings = bookInfo
        print(jsonmsg)
        jsonmsg = jsonmsg.replace("\t", "")

        jsondicts = json.loads(jsonmsg)
        jsondicts["copyright"] = copyright
        jsondicts["spon"] = spon
        jsondicts["author"] = authors
        jsondicts["author_institution"] = author_institution
        jsondicts["substrings"] = substrings
        jsonmsg = json.dumps(jsondicts, ensure_ascii=False)
        print(jsonmsg)
        sql = "insert into `ebookinfo` (`title`,`url`,`doi`,`jsonmsg`) values ('%s','%s','%s','%s') " \
              "on DUPLICATE key update `url`='%s',`jsonmsg`='%s'" % (
                  pymysql.escape_string(title), url1, doi, jsonmsg.replace("'", "\\\'"), url1,
                  jsonmsg.replace("'", "\\\'")
              )
        result_queue.put(sql)
        sql = "update `ebook` set stat=1 where `url`='%s'" % url1
        result_queue.put(sql)
        break


def readDBProxy():
    return getDataFromRedis(curPath, 'db.ini')


# 插入数据库
def InsertIntoDbFromList(ListSqls):
    for sql in ListSqls:
        if sql:
            myutil.ExeSqlToDB(sql, errExit=True)


class downThreadRun(MThreadingRun):

    def __init__(self, num, func):
        super(downThreadRun, self).__init__(num, func)

    def setTask(self, *args, **kwargs):
        if self.thread_pool.work_queue.empty():
            rows = SelectListFromDB()
            for row in rows:
                self.add_job(self.func, row, self.list_proxy)

    def dealresult(self, *args, **kwargs):
        InsertIntoDbFromList(self.results)

    def setProxy(self, proxysList=None):
        list_proxy = []
        rows = readDBProxy()
        for proxy in rows:
            if proxy:
                list_proxy.append(proxy)
        print("proxy leng is :" + str(len(list_proxy)))
        MThreadingRun.setProxy(self, list_proxy)

    def is_break(self):
        return True

def getTodayDate():
    """
    获取当前日期级时间
    如 20180810
    :return: string
    """
    return time.strftime('%Y-%m-%d 00:00:00', time.localtime(time.time()))

if __name__ == "__main__":
    # ebook 状态先清0
    func = down_page
    threadrun = downThreadRun(30, func)
    threadrun.setfunc(func)
    threadrun.run()
    sql = "update `ebookinfo` set stat=0 where update_time>'{}'".format(getTodayDate())
    myutil.ExeSqlToDB(sql, errExit=True)
    myutil.close()
